This notebook contains the key visualizations for the Task Mapping paper.

Very useful decision boundary plotting code from: https://mhahsler.github.io/Introduction_to_Data_Mining_R_Examples/book/classification-alternative-techniques.html#k-nearest-neighbors


decisionplot <- function(model, data, class_var, 
  predict_type = c("class", "prob"), resolution = 5 * 75) {
  # resolution is set to 75 dpi if the image is rendered  5 inces wide. 
  
  y <- data %>% pull(class_var)
  x <- data %>% dplyr::select(-all_of(class_var))
  
  # resubstitution accuracy
  prediction <- predict(model, x, type = predict_type[1])
  # LDA returns a list
  if(is.list(prediction)) prediction <- prediction$class
  prediction <- factor(prediction, levels = levels(y))
  
  cm <- confusionMatrix(data = prediction, reference = y)
  acc <- cm$overall["Accuracy"]
  
  # evaluate model on a grid
  r <- sapply(x[, 1:2], range, na.rm = TRUE)
  xs <- seq(r[1,1], r[2,1], length.out = resolution)
  ys <- seq(r[1,2], r[2,2], length.out = resolution)
  g <- cbind(rep(xs, each = resolution), rep(ys, time = resolution))
  colnames(g) <- colnames(r)
  g <- as_tibble(g)
  
  ### guess how to get class labels from predict
  ### (unfortunately not very consistent between models)
  cl <- predict(model, g, type = predict_type[1])
  
  # LDA returns a list
  if(is.list(cl)) { 
    prob <- cl$posterior
    cl <- cl$class
  } else
    try(prob <- predict(model, g, type = predict_type[2]))
  
  # we visualize the difference in probability/score between the 
  # winning class and the second best class.
  # don't use probability if predict for the classifier does not support it.
  max_prob <- 1
  try({
    max_prob <- t(apply(prob, MARGIN = 1, sort, decreasing = TRUE))
    max_prob <- max_prob[,1] - max_prob[,2]
  }, silent = TRUE) 
  
  cl <- factor(cl, levels = levels(y))
  
  g <- g %>% add_column(prediction = cl, probability = max_prob)
  
  ggplot(g, mapping = aes_string(
    x = colnames(g)[1],
    y = colnames(g)[2])) +
    geom_raster(mapping = aes(fill = prediction, alpha = probability)) +
     geom_contour(mapping = aes(z = as.numeric(prediction)), 
      bins = length(levels(cl)), size = .5, color = "black") +
    geom_point(data = data, mapping =  aes_string(
      x = colnames(data)[1],
      y = colnames(data)[2],
      shape = class_var), alpha = .7) + 
    scale_alpha_continuous(range = c(0,1), limits = c(0,1), guide = "none") +  
    labs(subtitle = paste("Training accuracy:", round(acc, 2)))
}

Load the Data

task_map <- read_csv('../task_map.csv')
Rows: 102 Columns: 25-- Column specification ----------------------------------------------------------------------------------------
Delimiter: ","
chr  (1): task
dbl (24): Q1concept_behav, Q3type_1_planning, Q4type_2_generate, Q6type_5_cc, Q7type_7_battle, Q8type_8_perf...
i Use `spec()` to retrieve the full column specification for this data.
i Specify the column types or set `show_col_types = FALSE` to quiet this message.

Fitting and Visualizing Models for the Task Map.

x <- combined_data %>% select(PC1, PC2, synergy, task)
train <- tasks_with_dv %>% select(PC1, PC2, synergy, task)
model <- train %>% svm(synergy ~ PC1 + PC2, data = ., kernel = "linear")

svmplot <- decisionplot(model, x, class_var = "synergy") + 
  geom_point(data = train, aes(x = PC1, y = PC2, shape = synergy), color = "darkolivegreen2", show.legend = F) +
  geom_label(data = train, aes(label = task ), nudge_y = 0.1, size = 3) +
  labs(title = "SVM (Linear Kernel)") +
  theme_minimal(base_size = 12)

svmplot
  
ggsave('svmplot_synthetic_data.png', width = 12, height = 5)

model <- train %>% knn3(synergy ~ PC1 + PC2, data = ., k = 1)

knnplot <- decisionplot(model, x, class_var = "synergy") +
  geom_point(data = train, aes(x = PC1, y = PC2, shape = synergy), color = "darkolivegreen2", show.legend = F) +
  geom_label(data = train, aes(label = task ), nudge_y = 0.1, size = 3) +
  labs(title = "kNN (1 neighbor)") + 
  theme_minimal(base_size = 12)

knnplot
  
ggsave('knnplot_synthetic_data.png', width = 12, height = 5)

LS0tCnRpdGxlOiAiUiBOb3RlYm9vayBmb3IgUGFwZXItUmVsYXRlZCBWaXN1YWxpemF0aW9ucyIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQoKVGhpcyBub3RlYm9vayBjb250YWlucyB0aGUga2V5IHZpc3VhbGl6YXRpb25zIGZvciB0aGUgVGFzayBNYXBwaW5nIHBhcGVyLgoKYGBge3J9CmxpYnJhcnkoZmFjdG9leHRyYSkKbGlicmFyeShOYkNsdXN0KQpsaWJyYXJ5KGNsdXN0ZXIpCmxpYnJhcnkocGxvdGx5KQpsaWJyYXJ5KGdncGxvdDIpCmxpYnJhcnkoY2FyZXQpICNmb3Iga25uCmxpYnJhcnkoZTEwNzEpICNmb3Igc3ZtCmxpYnJhcnkoZHBseXIpCmxpYnJhcnkodGlkeXZlcnNlKQpgYGAKClZlcnkgdXNlZnVsIGRlY2lzaW9uIGJvdW5kYXJ5IHBsb3R0aW5nIGNvZGUgZnJvbTogaHR0cHM6Ly9taGFoc2xlci5naXRodWIuaW8vSW50cm9kdWN0aW9uX3RvX0RhdGFfTWluaW5nX1JfRXhhbXBsZXMvYm9vay9jbGFzc2lmaWNhdGlvbi1hbHRlcm5hdGl2ZS10ZWNobmlxdWVzLmh0bWwjay1uZWFyZXN0LW5laWdoYm9ycwpgYGB7ciBkZWNpc2lvbnBsb3R9CgpkZWNpc2lvbnBsb3QgPC0gZnVuY3Rpb24obW9kZWwsIGRhdGEsIGNsYXNzX3ZhciwgCiAgcHJlZGljdF90eXBlID0gYygiY2xhc3MiLCAicHJvYiIpLCByZXNvbHV0aW9uID0gNSAqIDc1KSB7CiAgIyByZXNvbHV0aW9uIGlzIHNldCB0byA3NSBkcGkgaWYgdGhlIGltYWdlIGlzIHJlbmRlcmVkICA1IGluY2VzIHdpZGUuIAogIAogIHkgPC0gZGF0YSAlPiUgcHVsbChjbGFzc192YXIpCiAgeCA8LSBkYXRhICU+JSBkcGx5cjo6c2VsZWN0KC1hbGxfb2YoY2xhc3NfdmFyKSkKICAKICAjIHJlc3Vic3RpdHV0aW9uIGFjY3VyYWN5CiAgcHJlZGljdGlvbiA8LSBwcmVkaWN0KG1vZGVsLCB4LCB0eXBlID0gcHJlZGljdF90eXBlWzFdKQogICMgTERBIHJldHVybnMgYSBsaXN0CiAgaWYoaXMubGlzdChwcmVkaWN0aW9uKSkgcHJlZGljdGlvbiA8LSBwcmVkaWN0aW9uJGNsYXNzCiAgcHJlZGljdGlvbiA8LSBmYWN0b3IocHJlZGljdGlvbiwgbGV2ZWxzID0gbGV2ZWxzKHkpKQogIAogIGNtIDwtIGNvbmZ1c2lvbk1hdHJpeChkYXRhID0gcHJlZGljdGlvbiwgcmVmZXJlbmNlID0geSkKICBhY2MgPC0gY20kb3ZlcmFsbFsiQWNjdXJhY3kiXQogIAogICMgZXZhbHVhdGUgbW9kZWwgb24gYSBncmlkCiAgciA8LSBzYXBwbHkoeFssIDE6Ml0sIHJhbmdlLCBuYS5ybSA9IFRSVUUpCiAgeHMgPC0gc2VxKHJbMSwxXSwgclsyLDFdLCBsZW5ndGgub3V0ID0gcmVzb2x1dGlvbikKICB5cyA8LSBzZXEoclsxLDJdLCByWzIsMl0sIGxlbmd0aC5vdXQgPSByZXNvbHV0aW9uKQogIGcgPC0gY2JpbmQocmVwKHhzLCBlYWNoID0gcmVzb2x1dGlvbiksIHJlcCh5cywgdGltZSA9IHJlc29sdXRpb24pKQogIGNvbG5hbWVzKGcpIDwtIGNvbG5hbWVzKHIpCiAgZyA8LSBhc190aWJibGUoZykKICAKICAjIyMgZ3Vlc3MgaG93IHRvIGdldCBjbGFzcyBsYWJlbHMgZnJvbSBwcmVkaWN0CiAgIyMjICh1bmZvcnR1bmF0ZWx5IG5vdCB2ZXJ5IGNvbnNpc3RlbnQgYmV0d2VlbiBtb2RlbHMpCiAgY2wgPC0gcHJlZGljdChtb2RlbCwgZywgdHlwZSA9IHByZWRpY3RfdHlwZVsxXSkKICAKICAjIExEQSByZXR1cm5zIGEgbGlzdAogIGlmKGlzLmxpc3QoY2wpKSB7IAogICAgcHJvYiA8LSBjbCRwb3N0ZXJpb3IKICAgIGNsIDwtIGNsJGNsYXNzCiAgfSBlbHNlCiAgICB0cnkocHJvYiA8LSBwcmVkaWN0KG1vZGVsLCBnLCB0eXBlID0gcHJlZGljdF90eXBlWzJdKSkKICAKICAjIHdlIHZpc3VhbGl6ZSB0aGUgZGlmZmVyZW5jZSBpbiBwcm9iYWJpbGl0eS9zY29yZSBiZXR3ZWVuIHRoZSAKICAjIHdpbm5pbmcgY2xhc3MgYW5kIHRoZSBzZWNvbmQgYmVzdCBjbGFzcy4KICAjIGRvbid0IHVzZSBwcm9iYWJpbGl0eSBpZiBwcmVkaWN0IGZvciB0aGUgY2xhc3NpZmllciBkb2VzIG5vdCBzdXBwb3J0IGl0LgogIG1heF9wcm9iIDwtIDEKICB0cnkoewogICAgbWF4X3Byb2IgPC0gdChhcHBseShwcm9iLCBNQVJHSU4gPSAxLCBzb3J0LCBkZWNyZWFzaW5nID0gVFJVRSkpCiAgICBtYXhfcHJvYiA8LSBtYXhfcHJvYlssMV0gLSBtYXhfcHJvYlssMl0KICB9LCBzaWxlbnQgPSBUUlVFKSAKICAKICBjbCA8LSBmYWN0b3IoY2wsIGxldmVscyA9IGxldmVscyh5KSkKICAKICBnIDwtIGcgJT4lIGFkZF9jb2x1bW4ocHJlZGljdGlvbiA9IGNsLCBwcm9iYWJpbGl0eSA9IG1heF9wcm9iKQogIAogIGdncGxvdChnLCBtYXBwaW5nID0gYWVzX3N0cmluZygKICAgIHggPSBjb2xuYW1lcyhnKVsxXSwKICAgIHkgPSBjb2xuYW1lcyhnKVsyXSkpICsKICAgIGdlb21fcmFzdGVyKG1hcHBpbmcgPSBhZXMoZmlsbCA9IHByZWRpY3Rpb24sIGFscGhhID0gcHJvYmFiaWxpdHkpKSArCiAgICAgZ2VvbV9jb250b3VyKG1hcHBpbmcgPSBhZXMoeiA9IGFzLm51bWVyaWMocHJlZGljdGlvbikpLCAKICAgICAgYmlucyA9IGxlbmd0aChsZXZlbHMoY2wpKSwgc2l6ZSA9IC41LCBjb2xvciA9ICJibGFjayIpICsKICAgIGdlb21fcG9pbnQoZGF0YSA9IGRhdGEsIG1hcHBpbmcgPSAgYWVzX3N0cmluZygKICAgICAgeCA9IGNvbG5hbWVzKGRhdGEpWzFdLAogICAgICB5ID0gY29sbmFtZXMoZGF0YSlbMl0sCiAgICAgIHNoYXBlID0gY2xhc3NfdmFyKSwgYWxwaGEgPSAuNykgKyAKICAgIHNjYWxlX2FscGhhX2NvbnRpbnVvdXMocmFuZ2UgPSBjKDAsMSksIGxpbWl0cyA9IGMoMCwxKSwgZ3VpZGUgPSAibm9uZSIpICsgIAogICAgbGFicyhzdWJ0aXRsZSA9IHBhc3RlKCJUcmFpbmluZyBhY2N1cmFjeToiLCByb3VuZChhY2MsIDIpKSkKfQpgYGAKCiMgTG9hZCB0aGUgRGF0YQpgYGB7cn0KdGFza19tYXAgPC0gcmVhZF9jc3YoJy4uL3Rhc2tfbWFwLmNzdicpCmBgYAoKIyBQbG90IHRoZSBUYXNrIE1hcCBhbmQgb3RoZXIgUmVsYXRlZCBJbWFnZXMKCkRyYXcgdGhlIHRhc2sgbWFwIHVzaW5nIFBDQSAmIGNsdXN0ZXJpbmcKCkZpcnN0LCBydW4gdGhlIFBDQQpgYGB7ciwgZmlnLndpZHRoPTE2LCBmaWcuaGVpZ2h0PTV9CnNldC5zZWVkKDEpCgpwY2EgPC0gdGFza19tYXAgJT4lICNzZWxlY3QoLWNvbnRpbnVvdXNfcXVlc3Rpb25zKSAlPiUKICBzZWxlY3QoLXRhc2spICU+JQogIHByY29tcChjZW50ZXIgPSBUKQoKIyBnZXQgb3B0aW1hbCBudW1iZXIgb2YgY2x1c3RlcnMgLS0gInNpbGhvdWV0dGUiIG1ldGhvZApmdml6X25iY2x1c3QoeCA9IHBjYSR4LCBGVU5jbHVzdGVyID0gc3RhdHM6OmttZWFucywgbWV0aG9kID0gInNpbGhvdWV0dGUiKSArCiAgbGFicyhzdWJ0aXRsZSA9ICJTaWxob3VldHRlIG1ldGhvZCIpCgojIGdldCBvcHRpbWFsIG51bWJlciBvZiBjbHVzdGVycwpOYkNsdXN0KGRhdGEgPSBwY2EkeCwgZGlzdGFuY2UgPSAiZXVjbGlkZWFuIiwKICAgICAgICBtaW4ubmMgPSAyLCBtYXgubmMgPSAxNSwgbWV0aG9kID0gImttZWFucyIpCgprbWVhbnNfb3V0cHV0IDwtIHBjYSR4ICU+JSAKICBrbWVhbnMoY2VudGVycyA9IDMsIG5zdGFydCA9IDEwMCkKCmNvbWJpbmVkX2RhdGEgPC0gY2JpbmQodGFza19tYXAsCiAgICAgIHBjYSR4LCBmYWN0b3Ioa21lYW5zX291dHB1dCRjbHVzdGVyKSkgJT4lCiAgcmVuYW1lKGNsdXN0ZXIgPSBgZmFjdG9yKGttZWFuc19vdXRwdXQkY2x1c3RlcilgKQoKZnZpel9laWcocGNhKQpgYGAKClN0YW5kYXJkIFRhc2sgTWFwIEltYWdlIHdpdGggQWxsIExhYmVscwpgYGB7ciwgZmlnLndpZHRoPTE0LCBmaWcuaGVpZ2h0PTV9CnAgPC0gY29tYmluZWRfZGF0YSAlPiUKICBnZ3Bsb3QoYWVzKAogICAgeCA9IFBDMSwKICAgIHkgPSBQQzIsCiAgICBsYWJlbCA9IHRhc2ssCiAgICBmaWxsID0gY2x1c3RlcgogICkpICsgZ2VvbV9wb2ludCgpICsgZ2VvbV9sYWJlbChudWRnZV95ID0gMC4xLCBzaXplID0gNCkgKwogIAogICMrICwgYWxwaGE9MC4wNSkgKwojIGhpZ2hsaWdodHMgb25seSB0aGUgb25lcyBpbiB0aGUgc2VsZWN0ZWQgc2V0CiAgIyBnZW9tX2xhYmVsKAogICMgICBkYXRhID0gc3Vic2V0KGNvbWJpbmVkX2RhdGEsIHRhc2sgJWluJSBjKCJOQVNBIE1vb24gc3Vydml2YWwiLCAiRGVzZXJ0IHN1cnZpdmFsIikpLAogICMgICBhZXMoCiAgIyAgICAgeCA9IFBDMSwKICAjICAgICB5ID0gUEMyLAogICMgICAgIGxhYmVsID0gdGFzayAsCiAgIyAgICAgZmlsbCA9IGNsdXN0ZXIKICAjICAgKSwKICAjICAgbnVkZ2VfeSA9IDAuMSwKICAjICAgc2l6ZSA9IDIKICAjICkKIHRoZW1lX2xpZ2h0KGJhc2Vfc2l6ZSA9IDI0KQoKcCAjIHNob3cgdGhlIHBsb3QKCmdnc2F2ZShwbG90ID0gcCwgZmlsZW5hbWUgPSAnLi4vdGFzay1tYXAucG5nJykKYGBgCgpUYXNrIE1hcCBJbWFnZSBIaWdobGlnaHRpbmcgU3BlY2lmaWMgU3Vic2V0cyAoZm9yIElsbHVzdHJhdGl2ZSBQdXJwb3NlcykKYGBge3IsIGZpZy53aWR0aD0xNCwgZmlnLmhlaWdodD01fQojIEFuIGlsbHVzdHJhdGl2ZSBzZXQgdG8gZGlzcGxheQpkaXNwbGF5X3NldCA8LSBjKCdXcml0aW5nIHN0b3J5JywKICdBZHZlcnRpc2VtZW50IHdyaXRpbmcnLCAKICdEZXNlcnQgc3Vydml2YWwnLAogJ05BU0EgTW9vbiBzdXJ2aXZhbCcsCiAnVWx0aW1hdHVtIGdhbWUgKHZhcmlvdXMgdmVyc2lvbnMpJywKICdEaWN0YXRvciBnYW1lIGFuZCBpdHMgdmFyaWFudHMnLAogJ1ByaXNvbmVyXCdzIERpbGVtbWEgKHZhcmlvdXMgdmVyc2lvbnMpJywKICc5IERvdCBQcm9ibGVtJywKICdXb3JkIGNvbnN0cnVjdGlvbiBmcm9tIGEgc3Vic2V0IG9mIGxldHRlcnMnLAogJ1R5cGluZyBnYW1lJywKICdSYXZlbnMgTWF0cmljZXMnLAogJ0V1Y2xpZGVhbiB0cmF2ZWxpbmcgc2FsZXNwZXJzb24nCiApCgojIEEgc2V0IG9mIHRoZSB0YXNrcyB0aGF0IGFyZSBtb3N0IGRpZmZlcmVudAptYXhfZGlmZl9zZXQgPC0gYygnU2hvcHBpbmcgcGxhbicsCiAnTWluaW1hbCBHcm91cCBQYXJhZGlnbSAoc3R1ZHkgZGl2ZXJzaXR5KScsCiAnOSBEb3QgUHJvYmxlbScsCiAnV2hhYy1BLU1vbGUnLAogJ0J1bGxhcmQgSG91c2VzJywKICdQdXR0aW5nIGZvb2QgaW50byBjYXRlZ29yaWVzJywKICdDaGVja2VycycsCiAnUmVwcm9kdWNpbmcgYXJ0cycsCiAnQWxsb2NhdGluZyByZXNvdXJjZXMgdG8gcHJvZ3JhbXMnLAogJ0ltYWdlIHJhdGluZycsCiAnQXJpdGhtZXRpYyBwcm9ibGVtIDInKQoKIyBBIHNldCBvZiB0YXNrcyB0aGF0IGFyZSB0aGUgbW9zdCBzaW1pbGFyCm1pbl9kaWZmX3NldCA8LSBjKCdBcml0aG1ldGljIHByb2JsZW0gMScsCiAnRXVjbGlkZWFuIHRyYXZlbGluZyBzYWxlc3BlcnNvbicsCiAnQWJzdHJhY3QgZ3JpZCB0YXNrJywKICdNYXN0ZXJtaW5kJywKICdMb2dpYyBQcm9ibGVtJywKICdHdWVzc2luZyB0aGUgY29ycmVsYXRpb24nLAogJ1JhbmRvbSBkb3QgbW90aW9uJywKICdMZXR0ZXJzLXRvLW51bWJlcnMgcHJvYmxlbXMgKGNyeXB0b2dyYXBoeSknLAogJ0NvbXB1dGVyIG1hemUnLAogJ1JlY2FsbCBpbWFnZXMnLAogJ1JlY2FsbCBzdG9yaWVzJykKCiMgQSBzZXQgb2YgdGFza3MgdGhhdCBpbGx1c3RyYXRlcyBvcHBvcnR1bml0aWVzIHRvIGFkZCBuZXcgdGFza3MKZGlzcGxheV9saW1pdGF0aW9uc19zZXQgPC0gYygnUmVjYWxsIHdvcmQgbGlzdHMnLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICdIaWRkZW4gZmlndXJlcyBpbiBhIHBpY3R1cmUgKFJlY2FsbCBUYXNrKScsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ1JlY2FsbCBpbWFnZXMnLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICdSZWNhbGwgc3RvcmllcycsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ1JlY2FsbCB2aWRlb3MnLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICdXcml0aW5nIHN0b3J5JywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnQWR2ZXJ0aXNlbWVudCB3cml0aW5nJykKCgpncmFwaF9pbGx1c3RyYXRpdmVfcGxvdHMgPC0gZnVuY3Rpb24oZGlzcGxheXNldCwgZmlsZW5hbWUpewogIHAgPC0gY29tYmluZWRfZGF0YSAlPiUKICBnZ3Bsb3QoYWVzKAogICAgeCA9IFBDMSwKICAgIHkgPSBQQzIsCiAgICAjbGFiZWwgPSB0YXNrLAogICAgI2ZpbGwgPSBjbHVzdGVyCiAgICApKSArIGdlb21fcG9pbnQoYWVzKHNpemUgPSA0KSkgKwogICNnZW9tX3BvaW50KGFlcyhjb2xvciA9IGNsdXN0ZXIsIHNpemUgPSA0KSkgKwojaGlnaGxpZ2h0cyBvbmx5IHRoZSBvbmVzIGluIHRoZSBzZWxlY3RlZCBzZXQKZ2VvbV9wb2ludChkYXRhID0gc3Vic2V0KGNvbWJpbmVkX2RhdGEsIHRhc2sgJWluJSBkaXNwbGF5c2V0KSwgYWVzKHNpemUgPSA0KSwKICAgICAgICAgICBjb2xvciA9ICJmaXJlYnJpY2sxIikgKwpnZW9tX2xhYmVsKAogIGRhdGEgPSBzdWJzZXQoY29tYmluZWRfZGF0YSwgdGFzayAlaW4lIGRpc3BsYXlzZXQpLAogIGFlcygKICAgIHggPSBQQzEsCiAgICB5ID0gUEMyLAogICAgbGFiZWwgPSB0YXNrCiAgKSwKICBudWRnZV95ID0gMC4xNSwKICBzaXplID0gNAopICsKIHRoZW1lX21pbmltYWwoYmFzZV9zaXplID0gMTgpICsgdGhlbWUobGVnZW5kLnBvc2l0aW9uID0gIm5vbmUiKSAgCgpwCgpnZ3NhdmUocGxvdCA9IHAsIGZpbGVuYW1lID0gZmlsZW5hbWUsIHdpZHRoID0gMTQsIGhlaWdodCA9IDUpCn0KYGBgCgpgYGB7ciwgZmlnLndpZHRoPTE0LCBmaWcuaGVpZ2h0PTV9CmdyYXBoX2lsbHVzdHJhdGl2ZV9wbG90cyhkaXNwbGF5X2xpbWl0YXRpb25zX3NldCwgJy4uL2ltYWdlcy90YXNrLW1hcF93aXRoX25ld190YXNrX29wcG9ydHVuaXRpZXNfaGlnaGxpZ2h0ZWQucG5nJykKCmdyYXBoX2lsbHVzdHJhdGl2ZV9wbG90cyhtYXhfZGlmZl9zZXQsICcuLi9pbWFnZXMvdGFzay1tYXBfd2l0aF9tYXhfZGlmZl9oaWdobGlnaHRlZC5wbmcnKQoKIyBncmFwaF9pbGx1c3RyYXRpdmVfcGxvdHMobWluX2RpZmZfc2V0LCAnLi4vaW1hZ2VzL3Rhc2stbWFwX3dpdGhfbWluX2RpZmZfaGlnaGxpZ2h0ZWQucG5nJykKYGBgCgpDcmVhdGUgYSBjb29sIDNEIHZlcnNpb24KYGBge3J9CnBsb3RfbHkoCiAgeCA9IGNvbWJpbmVkX2RhdGEkUEMxLAogIHkgPSBjb21iaW5lZF9kYXRhJFBDMiwKICB6ID0gY29tYmluZWRfZGF0YSRQQzMsCiAgdHlwZSA9ICJzY2F0dGVyM2QiLAogIG1vZGUgPSAibWFya2VycyIsICMgY2FuIHVzZSBtb2RlID0gInRleHQiCiAgdGV4dCA9IGNvbWJpbmVkX2RhdGEkdGFzayAsCiAgY29sb3IgPSBjb21iaW5lZF9kYXRhJGNsdXN0ZXIKKQpgYGAKCkNyZWF0ZSBzeW50aGV0aWMgZGVwZW5kZW50IHZhcmlhYmxlIGJhc2VkIG9uIHRoZSBjbHVzdGVycwpgYGB7cn0KdGFza3Nfd2l0aF9kdiA8LSBzdWJzZXQoY29tYmluZWRfZGF0YSwgdGFzayAlaW4lIG1heF9kaWZmX3NldCkgJT4lCiAgbXV0YXRlKAogICAgc3luZXJneSA9IGFzLmZhY3RvcihpZmVsc2UoY2x1c3RlciA9PSAxIHwgY2x1c3RlciA9PSAyLCAxLCAwKSkKICApCmNvbWJpbmVkX2RhdGEgPC0gY29tYmluZWRfZGF0YSAlPiUKICBtdXRhdGUoCiAgICBzeW5lcmd5ID0gYXMuZmFjdG9yKGlmZWxzZShjbHVzdGVyID09IDEgfCBjbHVzdGVyID09IDIsIDEsIDApKQogICkKYGBgCgojIEZpdHRpbmcgYW5kIFZpc3VhbGl6aW5nIE1vZGVscyBmb3IgdGhlIFRhc2sgTWFwLgoKYGBge3J9CnggPC0gY29tYmluZWRfZGF0YSAlPiUgc2VsZWN0KFBDMSwgUEMyLCBzeW5lcmd5LCB0YXNrKQp0cmFpbiA8LSB0YXNrc193aXRoX2R2ICU+JSBzZWxlY3QoUEMxLCBQQzIsIHN5bmVyZ3ksIHRhc2spCm1vZGVsIDwtIHRyYWluICU+JSBzdm0oc3luZXJneSB+IFBDMSArIFBDMiwgZGF0YSA9IC4sIGtlcm5lbCA9ICJsaW5lYXIiKQoKc3ZtcGxvdCA8LSBkZWNpc2lvbnBsb3QobW9kZWwsIHgsIGNsYXNzX3ZhciA9ICJzeW5lcmd5IikgKyAKICBnZW9tX3BvaW50KGRhdGEgPSB0cmFpbiwgYWVzKHggPSBQQzEsIHkgPSBQQzIsIHNoYXBlID0gc3luZXJneSksIGNvbG9yID0gImRhcmtvbGl2ZWdyZWVuMiIsIHNob3cubGVnZW5kID0gRikgKwogIGdlb21fbGFiZWwoZGF0YSA9IHRyYWluLCBhZXMobGFiZWwgPSB0YXNrICksIG51ZGdlX3kgPSAwLjEsIHNpemUgPSAzKSArCiAgbGFicyh0aXRsZSA9ICJTVk0gKExpbmVhciBLZXJuZWwpIikgKwogIHRoZW1lX21pbmltYWwoYmFzZV9zaXplID0gMTIpCgpzdm1wbG90CiAgCmdnc2F2ZSgnc3ZtcGxvdF9zeW50aGV0aWNfZGF0YS5wbmcnLCB3aWR0aCA9IDEyLCBoZWlnaHQgPSA1KQpgYGAKCmBgYHtyfQptb2RlbCA8LSB0cmFpbiAlPiUga25uMyhzeW5lcmd5IH4gUEMxICsgUEMyLCBkYXRhID0gLiwgayA9IDEpCgprbm5wbG90IDwtIGRlY2lzaW9ucGxvdChtb2RlbCwgeCwgY2xhc3NfdmFyID0gInN5bmVyZ3kiKSArCiAgZ2VvbV9wb2ludChkYXRhID0gdHJhaW4sIGFlcyh4ID0gUEMxLCB5ID0gUEMyLCBzaGFwZSA9IHN5bmVyZ3kpLCBjb2xvciA9ICJkYXJrb2xpdmVncmVlbjIiLCBzaG93LmxlZ2VuZCA9IEYpICsKICBnZW9tX2xhYmVsKGRhdGEgPSB0cmFpbiwgYWVzKGxhYmVsID0gdGFzayApLCBudWRnZV95ID0gMC4xLCBzaXplID0gMykgKwogIGxhYnModGl0bGUgPSAia05OICgxIG5laWdoYm9yKSIpICsgCiAgdGhlbWVfbWluaW1hbChiYXNlX3NpemUgPSAxMikKCmtubnBsb3QKICAKZ2dzYXZlKCdrbm5wbG90X3N5bnRoZXRpY19kYXRhLnBuZycsIHdpZHRoID0gMTIsIGhlaWdodCA9IDUpCmBgYA==